package spark_from_scratch.mllib.regression

/**
 * @author andrew
 * @email zengjunjie1026@163.com
 * @date 2020/9/3 5:00 下午
 * @version 1.0
 */


import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
import org.apache.spark.mllib.linalg.Vectors
import java.io._
import org.apache.log4j.{Level, Logger}
object funpredict {
  def main(args: Array[String]) {
    val rand = new util.Random
    var one = rand.nextInt(900)
    var two = rand.nextInt(900)
    var three = rand.nextInt(900)
    var add = one+two+three
    for(i<-0 to 150){
      one = rand.nextInt(900)
      two = rand.nextInt(900)
      three = rand.nextInt(900)
      add = one+two+three
      //    println(add+","+one+" "+two+" "+three)
    }

    Logger.getLogger("org").setLevel(Level.ERROR)
    val path = new File(".").getCanonicalPath()
    System.getProperties().put("hadoop.home.dir", path);
    new File("./bin").mkdirs();
    new File("./bin/winutils.exe").createNewFile();

    val conf: SparkConf = new SparkConf().setAppName("LinearRegression").setMaster("local")
    val sc = new SparkContext(conf)


    // Load and parse the data
    val data = sc.textFile("F:\\testData\\spark\\funpredict.txt")
    val parsedData = data.map { line =>
      val parts = line.split(',')
      LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
    }.cache()
    //  parsedData.foreach(println)
    // Building the model
    val numIterations = 200
    val stepSize = 0.00001
    val model =  LinearRegressionWithSGD.train(parsedData, numIterations,stepSize)
    //   var lr = new LinearRegressionWithSGD().setIntercept(true)
    //   val model = lr.run(parsedData)


    //获取特征权重，及干扰特征
    println("weights:%s, intercept:%s".format(model.weights,model.intercept))
    //println(model.toPMML())
    // Evaluate model on training examples and compute training error
    val valuesAndPreds = parsedData.map { point =>
      val prediction = model.predict(point.features)
      (point.label, prediction)
    }
    println(model.predict(Vectors.dense(50,50,78)))
    //计算 均方误差
    val MSE = valuesAndPreds.map{case(v, p) => math.pow((v - p), 2)}.mean()
    println("training Mean Squared Error = " + MSE)


    // Save and load model
    //  model.save(sc, "myModelPath")
    //  val sameModel = LinearRegressionModel.load(sc, "myModelPath")




  }


}
